package classification.elemListRepr;

import java.io.File;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Set;
import java.util.StringTokenizer;

import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;

import classification.Dictionary;

import edu.udo.cs.wvtool.external.Stopwords;
import edu.udo.cs.wvtool.generic.stemmer.PorterStemmerWrapper;
import edu.udo.cs.wvtool.generic.tokenizer.SimpleTokenizer;
import edu.udo.cs.wvtool.main.WVTDocumentInfo;

import preprocess.Arith;
import preprocess.DBWrapper;

import utils.Configuration;
import static utils.Configuration.*;

/**
 * This class is used to generate the element list represent for all xml
 * document, and each element list item is one element that occurs in some xml
 * documents and different from each other.
 * 
 * @author eDisOn
 * 
 */
public class ElemListReprGenerator
{
    /* 路径有关的静态常量 */
    private static final String   ROOT                  = RootPath;
    private static final String   XML_ELEM_LIST_DBNAME  = Configuration.SIMILARITY.tempalte_database;
    private static final String   FEATURE_TEMPLATE_NAME = Configuration.SIMILARITY.feature_template;

    /* 通过遍历找到的ElemListRepr模板 */
    private int                   totalXMLFileNum       = 0;
    private Dictionary            dict                  = null;
    private static XMLElementList xmlOverallElemList    = null;
    private static XMLElementList xmlFeatureElemList    = null;

    public ElemListReprGenerator()
    {
        dict = Dictionary.getInstance();
    }

    /**
     * 一方面创造由所有出现过的element作为维度的，总的xmltemplate表示。
     * 另一方面根据element出现的频繁程度<b>proportion</b>进行特征元素的选择， 从而得到总的xml 特征向量空间
     * 
     * @param trainingFiles
     *            待训练的文件
     * 
     * @param proportion
     *            指定待选出的element出现的频繁程度
     */
    public void ELTemplateGenerator(final File[] trainingFiles,
            double proportion)
    {
        try
        {
            ElemListReprGenerator.xmlOverallElemList = new XMLElementList(
                    "OVERALL_TEMPLATE");

            /* get all xml file list */
            File[] XMLFileList = trainingFiles;
            this.totalXMLFileNum = XMLFileList.length;

            /*
             * generate element list representation by traverse all xml files in
             * xml file list
             */
            SAXReader saxReader = new SAXReader();
            Document document = null;
            for (File XMLFile : XMLFileList)
            {
                document = saxReader.read(XMLFile);
                this.parseXMLDoc(document);
            }

            /* Feature element selection */
            ElemListReprGenerator.xmlFeatureElemList = this
                    .featureElemSelect(proportion);
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }

    }

    /**
     * 解析每一个训练集中的xml document，从而得到总的 xml template
     * 
     * @param document
     *            xml document对象
     */
    private void parseXMLDoc(Document document)
    {
        try
        {
            Element XMLRoot = document.getRootElement();

            XMLElement matchElem = ElemListReprGenerator.xmlOverallElemList
                    .contains(XMLRoot.getName());
            if (matchElem == null)
            {
                XMLElement newElement = new XMLElement(XMLRoot.getName(), 1);
                ElemListReprGenerator.xmlOverallElemList.addXMLElem(newElement);
            }
            else
            {
                matchElem.increaseFreq();
            }

            this.exploreNode(XMLRoot);
        }
        catch (Exception ex)
        {
            ex.printStackTrace();
        }
    }

    /**
     * 递归遍历每一个节点，得到所有出现的element
     * 
     * @param elem
     *            当前递归到得element对象
     */
    private void exploreNode(Element elem)
    {
        for (Iterator it = elem.elementIterator(); it.hasNext();)
        {
            Element child = (Element) it.next();
            String childElemName = child.getName();

            XMLElement matchElem = ElemListReprGenerator.xmlOverallElemList
                    .contains(childElemName);
            if (matchElem == null)
            {
                XMLElement newElement = new XMLElement(childElemName, 1);
                ElemListReprGenerator.xmlOverallElemList.addXMLElem(newElement);
            }
            else
            {
                matchElem.increaseFreq();
            }

            this.exploreNode(child);
        }
    }

    /**
     * 根据element出现的频繁程度<b>proportion</b>进行特征元素的选择，从而得到总的xml特征向量空间模板
     * 
     * @param proportion
     *            指定待选出的element出现的频繁程度
     * 
     * @return 选取得到的XMLElementList的模板
     */
    private XMLElementList featureElemSelect(double proportion)
    {
        XMLElementList ElemList = new XMLElementList(FEATURE_TEMPLATE_NAME);
        for (Iterator<XMLElement> it = ElemListReprGenerator.xmlOverallElemList
                .Iterator(); it.hasNext();)
        {
            XMLElement elem = it.next();
            if (elem.getFrequency() >= this.totalXMLFileNum * proportion)
            {
                ElemList.addXMLElem(elem);
            }
        }
        return ElemList;
    }

    /**
     * 将一个xml document以该ElemListReprGenerator对象中为xmlFeatureElemList模板的
     * XMLElementList表示 ，每一个特征元素作为一个维度
     * 
     * @param document
     *            待表示的xml document
     * @param docID
     *            待表示的xml document的identifier
     * @return 该xml document对应的XMLElementList对象
     */
    public XMLElementList XMLELReprGenerator(Document document, String docID)
    {
        /* 按照特征模板创建模板XMLElementList */
        XMLElementList XMLELReprList = new XMLElementList(docID);
        XMLELReprList.copyXMLElemList(ElemListReprGenerator.xmlFeatureElemList);

        /* 生成该文档的XMLElementList */
        Element root = document.getRootElement();
        this.ELReprExploreNode(XMLELReprList, root);

        /* VSM modeling */
        XMLELReprList = this.VSMModeling(XMLELReprList);

        /* Calculate tfidf */
        XMLELReprList = this.calcTfidf(XMLELReprList);

        return XMLELReprList;
    }

    /**
     * 使用后序遍历的方法得到s各个元素位置处的text，其中如果某元素包含text，而该元素本身不属于向量空间的某一维度，
     * 则将该元素的text内容添加到其父节点中
     * 
     * @param XMLELReprList
     *            xml document对应的空间向量
     * @param elem
     *            正在分析的element元素
     */
    private void ELReprExploreNode(XMLElementList XMLELReprList, Element elem)
    {
        /* LRN traversal */
        for (Iterator it = elem.elementIterator(); it.hasNext();)
        {
            this.ELReprExploreNode(XMLELReprList, (Element) it.next());
        }

        String elemName = elem.getName();
        boolean flag = false;

        for (Iterator<XMLElement> it = XMLELReprList.Iterator(); it.hasNext()
                && (!flag);)
        {
            XMLElement xmlElem = it.next();
            if (xmlElem.getXMLElemName().equals(elemName))
            {
                xmlElem.appendText(elem.getTextTrim());
                flag = true;
                break;
            }
        }
        if (!flag)
        {
            Element parent = elem.getParent();

            if (parent != null)
            {
                String text = elem.getTextTrim();
                parent.addText(text + " ");
            }
        }
    }

    private XMLElementList VSMModeling(XMLElementList XMLELReprList)
    {
        int totalVocNum = 0;
        /*
         * vocabularyList is used to record all vocabularies in this XML file(in
         * XMLElementList representation)
         */
        HashMap<String, Double> vocabularyList = new HashMap<String, Double>();

        /*
         * 对XMLElementList里每一个Element的文本内容利用wvtool进行分词，stemming，去除停用词，并自后进行统计词频操作
         */
        StringReader source = null;
        SimpleTokenizer tokenizer = null;
        WVTDocumentInfo info = new WVTDocumentInfo("", "", "", "");
        PorterStemmerWrapper pStemmer = new PorterStemmerWrapper();

        for (Iterator it = XMLELReprList.Iterator(); it.hasNext();)
        {
            XMLElement elem = (XMLElement) it.next();
            String textContent = elem.getText();
            String newContent = "";
            int elementVocNum = 0;

            /* tokenize the text content in each XML Element */
            source = new StringReader(textContent);
            tokenizer = new SimpleTokenizer();
            tokenizer.tokenize(source, info);

            /* word filter, stemming and hashMap content */
            String originalterm = "";
            String stmmedTerm = "";
            while (tokenizer.hasMoreTokens())
            {
                originalterm = tokenizer.nextToken();
                stmmedTerm = "";
                if (!Stopwords.isStopword(originalterm))
                {
                    stmmedTerm = pStemmer.getBase(originalterm);
                    if (!vocabularyList.containsKey(stmmedTerm))
                    {
                        vocabularyList.put(stmmedTerm, 1.0);
                    }
                    else
                    {
                        double value = vocabularyList.get(stmmedTerm);
                        vocabularyList.put(stmmedTerm, value + 1);
                    }
                }
                if (stmmedTerm != "")
                {
                    newContent += stmmedTerm + "\t";
                    totalVocNum++;
                    elementVocNum++;
                }
            }

            /*
             * replace processed text content which has been tokenized and
             * stemmed to this element object
             */
            elem.replaceText(newContent);
            elem.setVocNum(elementVocNum);
        }

        /* vector creator */
        StringTokenizer stk = null;
        for (Iterator it = XMLELReprList.Iterator(); it.hasNext();)
        {
            XMLElement elem = (XMLElement) it.next();
            String textContent = elem.getText();
            stk = new StringTokenizer(textContent, "\t");
            while (stk.hasMoreTokens())
            {
                String voc = stk.nextToken();
                elem.addVocabulary(voc);
            }
        }

        /*
         * update total vocabulary number information and return this
         * XMLElementList object
         */
        XMLELReprList.setTotalVocNum(totalVocNum);
        return XMLELReprList;
    }

    private XMLElementList calcTfidf(XMLElementList XMLELReprList)
    {
        int idf = 0;
        double counts = 0.0;
        double tfidf = 0.0;
        double sum = 0.0;
        double normFactor = 0.0;
        int totalDocumentNum = dict.getDocumentNum();
        for (Iterator<XMLElement> it = XMLELReprList.Iterator(); it.hasNext();)
        {
            /* calculate tfidf value for each XML element */
            idf = 0;
            counts = 0.0;
            tfidf = 0.0;
            sum = 0.0;
            HashMap<String, Double> vocubularyList = it.next()
                    .getVocabularyList();
            Set<String> keySet = vocubularyList.keySet();
            for (String key : keySet)
            {
                idf = dict.getIDF(key);
                counts = vocubularyList.get(key);
                if (idf != 0)
                {
                    tfidf = (1 + Math.log(counts))
                            * Math.log(totalDocumentNum / ((double) idf));
                    tfidf = Arith.round(tfidf, 8);

                    /* update tfidf value */
                    vocubularyList.put(key, tfidf);

                    /* add this term's tfidf to sum */
                    sum += Math.pow(tfidf, 2);
                }
            }

            normFactor = Math.sqrt(sum);
            if (normFactor == 0)
            {
                continue;
            }

            tfidf = 0.0;
            for (String key : keySet)
            {
                /* normalize the tfidf value */
                tfidf = vocubularyList.get(key);
                tfidf = Arith.round(tfidf / normFactor, 8);
                vocubularyList.put(key, tfidf);
            }
        }

        return XMLELReprList;
    }

    /**
     * Store the XMLElementList object to database
     * 
     * @param XMLELReprList
     *            The XMLElementList object to be stored
     * @return true if succeed, false otherwise
     */
    public boolean storeXMLEListTemplate()
    {
        try
        {
            DBWrapper dbWrapper = new DBWrapper();

            if (dbWrapper.exists(XML_ELEM_LIST_DBNAME))
            {
                dbWrapper.insert(ElemListReprGenerator.xmlFeatureElemList,
                        XML_ELEM_LIST_DBNAME);
            }
            else
            {
                dbWrapper.createDB(XML_ELEM_LIST_DBNAME);
                dbWrapper.insert(ElemListReprGenerator.xmlFeatureElemList,
                        XML_ELEM_LIST_DBNAME);
            }
            return true;
        }
        catch (Exception ex)
        {
            return false;
        }
    }

    public XMLElementList loadXMLEListTemplate()
    {
        DBWrapper dbWrapper = new DBWrapper();

        /* 在数据库中查找是否有 XMLElementList 模板已经存入数据库，如果有的话将数据库中的导入 */
        XMLElementList xel = dbWrapper.queryXMLElemList(XML_ELEM_LIST_DBNAME,
                FEATURE_TEMPLATE_NAME);
        if (xel != null)
        {
            ElemListReprGenerator.xmlFeatureElemList = xel;
        }

        /* 如果数据库中没有该模板，则创建只包含"article"的矩阵模板返回 */
        else
        {
            XMLElementList template = new XMLElementList("template");
            template.addXMLElem(new XMLElement("article"));
            ElemListReprGenerator.xmlFeatureElemList = template;
        }
        return ElemListReprGenerator.xmlFeatureElemList;
    }

    public void setTemplate(XMLElementList template)
    {
        ElemListReprGenerator.xmlFeatureElemList = template;
    }

    public XMLElementList getTemplate()
    {
        return ElemListReprGenerator.xmlFeatureElemList;
    }

//    public static void main(String[] args)
//    {
//        ElemListReprGenerator generator = new ElemListReprGenerator();
//        XMLElementList template = new XMLElementList(Configuration
//                .getAttribute("classification", "FEATURE_TEMPLATE_NAME"));
//        template.addXMLElem(new XMLElement("body"));
//        template.addXMLElem(new XMLElement("name"));
//        template.addXMLElem(new XMLElement("p"));
//        template.addXMLElem(new XMLElement("section"));
//        template.addXMLElem(new XMLElement("title"));
//        template.addXMLElem(new XMLElement("figure"));
//        template.addXMLElem(new XMLElement("caption"));
//        template.addXMLElem(new XMLElement("item"));
//        generator.setTemplate(template);
//        generator.storeXMLEListTemplate();
//    }
}
